# -*- coding:utf8 -*-
#!/usr/bin/env python
from time import sleep
import nltk, re, math,threading
from django.utils.encoding import smart_str, smart_unicode

from django.core.management.base import BaseCommand, CommandError
from feedsaver.saver.models import *

def loop(self,subject):
	global nglobal
	stopwords = nltk.corpus.stopwords.words('portuguese')
	#stemmer = nltk.stem.RSLPStemmer()

	ns = NewsArchive.objects.filter(subject=subject)
	words = []

	for n in ns:
		nglobal = nglobal+1
		#command.stdout.write("N de noticias: "+str(nglobal)+'\n')
	
		text = n.text

		text = text.split(' ')

		#Removing stopwords
		for word in text:
			word = smart_str(word, encoding='utf-8', strings_only=False, errors='strict')

			if (len(word) > 2):
				if word not in stopwords:	
					words.append(word)


	return words

class loop_brasil ( threading.Thread ):
    def run ( self ):
    	global allwords1
        allwords1 = loop(self,'brasil')

class loop_mundo ( threading.Thread ):
    def run ( self ):
    	global allwords2
        allwords2 = loop(self,'mundo')

class loop_ciencia ( threading.Thread ):
    def run ( self ):
    	global allwords3
        allwords3 = loop(self,'ciencia')

class loop_tecnologia ( threading.Thread ):
    def run ( self ):
    	global allwords4
        allwords4 = loop(self,'tecnologia')

class loop_entretenimento( threading.Thread ):
    def run ( self ):
    	global allwords5
        allwords5 = loop(self,'entretenimento')

class loop_economia( threading.Thread ):
    def run ( self ):
    	global allwords6
        allwords6 = loop(self,'economia')


def mcw(self):
	global allwords1
	global allwords2
	global allwords3
	global allwords4
	global allwords5
	global allwords6
	#Monta o array de features: 2000 palavras mais comuns de TODOS os textos

	#Faz um loop por todos os textos de um assunto e retorna um array com todas as palavras
	loop_brasil().start()
	loop_mundo().start()
	loop_ciencia().start()
	loop_tecnologia().start()
	loop_entretenimento().start()
	loop_economia().start()

	while (threading.activeCount() > 1):
		sleep(1)

	allwords = allwords1 + allwords2 + allwords3 + allwords4 + allwords5 + allwords6

	vfa = VectorFeatures.objects.all()
	if (vfa.count() != 0):
		for v in vfa:
			v.delete()
	
	vf = VectorFeatures()
	vf.type = 'principal'
	array = []
	array2 = []

	# Verifica as 2000 palavras mais ocorrentes em todos os textos
	# Salva a frequencia delas em todos os textos para calculo do IDF posteriormente
	fd = nltk.FreqDist(w for w in allwords)
	for word in fd.keys()[:2000]:
		array.append(word)
		array2.append(str(fd[word]))

		#self.stdout.write(word+' '+str(fd[word])+'\n')

	vf.feature = ','.join(array)
	vf.values = ','.join(array2)
	vf.save()

def subject_features(self,subject,vfeatures):
	global n2global

	ns = NewsArchive.objects.filter(subject=subject)

	nscount = NewsArchive.objects.filter(subject=subject).count()

	totalnews = NewsArchive.objects.all().count()

	tmps = VectorFeatures.objects.filter(type=subject)

	principal = VectorFeatures.objects.get(type='principal').values
	principal = principal.split(',')

	for t in tmps:
		t.delete()

	#Vetor de treinamento com metade das notícias
	for n in ns[:(nscount/2)]:
		n2global = n2global + 1
		self.stdout.write(str(n2global)+'\n')

		lista = []
		vf = VectorFeatures()
		vf.type = subject

		vtext = n.text.split(' ')
		
		for i in vfeatures:
			lista.append(0)	

		k1 = 0
		for i in vfeatures:

			k2 = 0
			for j in vtext:
				if i == j:
					# Calcula a frequencia das 2000 palavras nesse texto
					lista[k1] = str(int(lista[k1]) + 1)	
			
				k2 = k2+1	

			k1 = k1+1

		#TF-IDF
		lenvtext = len(vtext)
		for i in lista:
			#tf
			tf = (int(i) / lenvtext)

			#self.stdout.write(principal[int(i)])
			idf = math.log10(totalnews/int(principal[int(i)]))

			i = tf*idf

		vf.values = ','.join( map( str, lista ))
		vf.save()

		#self.stdout.write(vf.values)
		#self.stdout.write('\n')

	return

class Command(BaseCommand):
    args = 'no args'
    help = 'Extract info from CPR'
    
    def handle(self, *args, **options):
    	global nglobal
    	global allwords
    	global allwords1
    	global allwords2
    	global allwords3
    	global allwords4
    	global allwords5
    	global allwords6

    	nglobal = 0
    	allwords = []
    	allwords1 = []
    	allwords2 = []
    	allwords3 = []
    	allwords4 = []
    	allwords5 = []
    	allwords6 = []

    	global command
    	command = self

        mcw(command)

        vfeatures = VectorFeatures.objects.get(type='principal').feature.split(',')

        global n2global
        n2global = 0
        subject_features(self,'brasil',vfeatures)
        subject_features(self,'mundo',vfeatures)
        subject_features(self,'economia',vfeatures)
        subject_features(self,'tecnologia',vfeatures)
        subject_features(self,'ciencia',vfeatures)
        subject_features(self,'entretenimento',vfeatures)

        self.stdout.write('Fim da montagem de vetores de características\n')

        return